import re
import threading

from bs4 import BeautifulSoup
from util import utils, httpUtil

path = "src/video"
baseUrl = "https://www.sex.com"

def getUrlList(html):
    list = []
    t1 = BeautifulSoup(html, 'html.parser').find_all(name='a',attrs={"class":"image_wrapper"})
    for url in t1:
            list.append(url.get("href"))
    return list

def reForUrl(s):
    s1 = re.findall(r'src: \'\/(.+?)\'', s, re.M)
    return s1[0]

# 获取重定向之后的URL
def getVideoUrl(html):
    t1 = BeautifulSoup(html, 'html.parser').findAll(name='script');
    for script in t1:
        if str(script).__contains__(" videojs("):
            url = baseUrl + "/" + reForUrl(str(script))
            realurl = httpUtil.get_real_url(url,1)
            print("url = [" + url+"]real_url = [" + realurl+"]")
            return realurl


def downloadv(url):
    html = utils.getHtmlUtf_8(baseUrl + url)
    url = getVideoUrl(html)
    utils.saveVideo(url, path)

def getVideoURLList(urlList):
    list = []
    for url in urlList:
        t = threading.Thread(target=downloadv, args=(url,))
        t.start()
    return list


def download(url):
    print(url)
    html = utils.getHtmlUtf_8(url)
    urlList = getUrlList(html)
    getVideoURLList(urlList)

def start(pageIndex):
    if pageIndex == 1:
        download("https://www.sex.com/videos/")
    else:
        download("https://www.sex.com/videos/?page=" + str(pageIndex))

if __name__ == "__main__":
    n = 0
    while 0<= n < 6:
        n = n + 1
        t = threading.Thread(target=start,args=(n,))
        t.start()